package com.bj58.heatmap.main;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.bj58.groupbuy.util.PathUtil;
import com.bj58.heatmap.db.CityDao;
import com.bj58.heatmap.model.DispLocal;

/**
 * 抓取城市首页
 * @author zhaoxiang
 */

public class WebCrawler {
	private static String htmlPath=null;
	private static String imgPath=null;
	private static String htmlPath_new=null;
	public static String htmlPath_save_test=null;

	static{
		String path = PathUtil.getWFNamespaceFolder()+ "/html_savePath.properties";
		Properties p = new Properties();
		try {
			FileInputStream fis = new FileInputStream(new File(path));
			p.load(fis);
			fis.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		htmlPath_new=p.getProperty("html_Path");
		htmlPath_save_test=p.getProperty("htmlPath_save_test");
		htmlPath=p.getProperty("html_Path_r");
		imgPath=p.getProperty("js_img_Path");
	}
	
	public static void main(String args[]) {
		String dateFolder = new SimpleDateFormat("yyyyMMdd").format(new Date());
		// 创建文件夹
		File savePath = new File(htmlPath+dateFolder);
		if (!savePath.exists())
			savePath.mkdirs();
		File js_img = new File(imgPath+dateFolder );
		if (!js_img.exists()) {
			js_img.mkdirs();
		}
		WebCrawler crawer = new WebCrawler();
		List<DispLocal> cityList = CityDao.getDispLocalList();
		System.out.println("共查询到"+cityList.size()+"个城市！");
		for (DispLocal local : cityList) {
			//System.out.println(local.getList_name());
			crawer.startDownload(local.getCity_name_en(),savePath.getPath(),js_img.getPath());
			crawer.startDownload_M(local.getCity_name_en(),savePath.getPath(),js_img.getPath());
		}
//		String[] page = { "bj", "sh"};
//		for (String s : page){
//			crawer.startDownload(s,savePath.getPath(),js_img.getPath());
//			crawer.startDownload_M(s,savePath.getPath(),js_img.getPath());
//		}
		System.out.println("WebCrawler complete！");
		System.exit(0);
	}

	public void startDownload(String city ,String htmlFolder,String js_img_folder) {
		String html = "";// 存放网页HTML源代码
		String url = "http://" +city + ".58.com";
		// 下载网页html
		html = getHTML(url);
		
		if (html == null)
			return;
/*
		String date=htmlFolder.substring(htmlFolder.length()-9,htmlFolder.length()-4)+"-"+htmlFolder.substring(htmlFolder.length()-4,htmlFolder.length()-2)+"-"+htmlFolder.substring(htmlFolder.length()-2,htmlFolder.length());
		添加label，指向该城市M端的链接
		buttonBar = "<div align=\"center\">\n<br/>\n<button type=\"button\">\n"
			+ " <a href=\"/heatmap/getPage?selectpvtype=m&selectdate="+date+"&selectedcityname="+city+"\" target=\"_self\">切换到m端</a>\n" + "</button>\n</div>";
		html=html.replaceFirst("<div\\s{1,}(.*?)id=('|\")header\\2(.*?)>", "<div $1id=\"header\"$3>" + colorBar+buttonBar);
*/		
		// 下载js脚本和图片
		downloadResource(html, js_img_folder);
		// 保存网页HTML到文件
		try {
			BufferedWriter bw = new BufferedWriter(new FileWriter(htmlFolder
					+ "/" + city + ".html"));
			bw.write(html);
			bw.flush();
			bw.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}


	public void startDownload_M(String cityname ,String htmlFolder,String js_img_folder) {
		String html = "";// 存放网页HTML源代码
		String url = "http://m.58.com/" +cityname+"/?refrom=wap";
		// 下载网页html
		html = getHTML(url);
		
		if (html == null)
			return;
		
/*		
		String date=htmlFolder.substring(htmlFolder.length()-9,htmlFolder.length()-4)+"-"+htmlFolder.substring(htmlFolder.length()-4,htmlFolder.length()-2)+"-"+htmlFolder.substring(htmlFolder.length()-2,htmlFolder.length());
		添加button，指向该城市M端的链接
		buttonBar = "<div align=\"center\">\n<br/>\n<button type=\"button\">\n"
			+ " <a href=\"/heatmap/getPage?selectpvtype=pc&selectdate="+date+"&selectedcityname="+cityname+"\" target=\"_self\">切换到pc端</a>\n" + "</button>\n</div>";
		html=html.replaceFirst("(<body.*?>)","$1" + colorBar_M+buttonBar);
*/
		// 下载js脚本和图片
		downloadResource_M(html, js_img_folder);
		// 保存网页HTML到文件
		try {
			BufferedWriter bw = new BufferedWriter(new FileWriter(htmlFolder
					+ "/" + cityname + "_M.html"));
			bw.write(html);
			bw.flush();
			bw.close();
		} catch (IOException e) {
			e.printStackTrace();
			return;
		}
	}


	/**
	 * 根据指定的URL下载html代码
	 * 
	 * @param pageURL
	 * @return
	 */
	public String getHTML(String pageURL) {
		String line_separator = System.getProperty("line.separator");
		StringBuffer pageHTML = new StringBuffer();
		try {
			URL url = new URL(pageURL);
			HttpURLConnection connection = (HttpURLConnection) url
					.openConnection();
			BufferedReader br = new BufferedReader(new InputStreamReader(
					connection.getInputStream()));
			String line = null;
			while ((line = br.readLine()) != null) {
				pageHTML.append(line);
				pageHTML.append(line_separator);
			}
			connection.disconnect();
			// fileWriter.close();
		} catch (Exception e) {
			System.err.println(pageURL + "抓取失败！");
			return null;
		}
		return pageHTML.toString();
	}

	/**
	 * 下载脚本，图片
	 * 
	 * @param html
	 * @param resPath
	 * @return
	 */
	public String downloadResource(String html, String resPath) {
		ArrayList<String> srcList = new ArrayList<String>();
		String srcReg = "http://([^<>'+()]*?)\\.(js|css|jpeg|jpg|png|gif|bmp|swf)";
		Pattern p = Pattern.compile(srcReg);
		Matcher m = p.matcher(html);
		StringBuffer sb = new StringBuffer();
		while (m.find()) {
			String src = m.group();
			srcList.add(src);
			int i = src.indexOf("?");
			if (i != -1)
				src = src.substring(0, i);
			downloadFile(src,
					resPath + "/" + src.substring(src.lastIndexOf("/") + 1));
		}
		return sb.toString();
	}
	
	/**
	 * 下载脚本，图片
	 * 
	 * @param html
	 * @param resPath
	 * @return
	 */
	public String downloadResource_M(String html, String resPath) {
		ArrayList<String> srcList = new ArrayList<String>();
		String srcReg = "http://([^<>'+()]*?)\\.(js|css|jpeg|jpg|png|gif|bmp|swf)";
		Pattern p = Pattern.compile(srcReg);
		Matcher m = p.matcher(html);
		StringBuffer sb = new StringBuffer();
		while (m.find()) {
			String src = m.group();
			srcList.add(src);
			int i = src.indexOf("?");
			if (i != -1)
				src = src.substring(0, i);
			downloadFile(src,resPath + "/M_" + src.substring(src.lastIndexOf("/") + 1));
		}
		return sb.toString();
	}
	
	/**
	 * 根据URL下载某个文件
	 * 
	 * @param fileURL
	 *            下载地址
	 * @param filePath
	 *            存放的路径
	 */
	public void downloadFile(String fileURL, String filePath) {
		File file = new File(filePath);
		if (!file.exists())
			try {
				
//				StringBuffer sb = new StringBuffer();
				URL url = new URL(fileURL);
				HttpURLConnection connection = (HttpURLConnection) url
						.openConnection();
				byte[] buffer = new byte[1024];
				InputStream is = connection.getInputStream();
				file.createNewFile();
				FileOutputStream fos = new FileOutputStream(file);
				int len = 0;
				while ((len = is.read(buffer)) != -1)
					fos.write(buffer, 0, len);
				fos.close();
				is.close();
				connection.disconnect();
			} catch (IOException e) {
				e.printStackTrace();
			}
	}

	public HashSet<String> extactURLS(String mainUrl) {
		HashSet<String> urls = new HashSet<String>();
		String html = getHTML(mainUrl);
		Pattern p = Pattern.compile("href=\"http://([^<>]*?)\\.58\\.com/\"");
		Matcher m = p.matcher(html);
		m.find();
		while (m.find()) {
			urls.add("http://" + m.group(1) + ".58.com");
		}
		System.out.println(urls.size());
		return urls;
	}

	public static String getHtmlPath() {
		return htmlPath;
	}

	public static void setHtmlPath(String htmlPath) {
		WebCrawler.htmlPath = htmlPath;
	}

	public static String getImgPath() {
		return imgPath;
	}

	public static void setImgPath(String imgPath) {
		WebCrawler.imgPath = imgPath;
	}

	public static String getHtmlPath_new() {
		return htmlPath_new;
	}

	public static void setHtmlPath_new(String htmlPath_new) {
		WebCrawler.htmlPath_new = htmlPath_new;
	}
	
	
}
